/*
 * #!
 * Ontopia Engine
 * #-
 * Copyright (C) 2001 - 2013 The Ontopia Project
 * #-
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 *      http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * !#
 */

package net.ontopia.topicmaps.query.impl.rdbms;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;

import net.ontopia.persistence.proxy.QueryIF;
import net.ontopia.persistence.proxy.RDBMSStorage;
import net.ontopia.persistence.query.jdo.JDOAnd;
import net.ontopia.persistence.query.jdo.JDOEvaluator;
import net.ontopia.persistence.query.jdo.JDOExpressionIF;
import net.ontopia.persistence.query.jdo.JDONot;
import net.ontopia.persistence.query.jdo.JDOOr;
import net.ontopia.persistence.query.jdo.JDOParameter;
import net.ontopia.persistence.query.jdo.JDOQuery;
import net.ontopia.persistence.query.jdo.JDOValueIF;
import net.ontopia.persistence.query.jdo.JDOVariable;
import net.ontopia.persistence.query.jdo.JDOVisitorIF;
import net.ontopia.persistence.query.sql.SQLGeneratorIF;
import net.ontopia.topicmaps.query.core.InvalidQueryException;
import net.ontopia.topicmaps.query.core.ParsedQueryIF;
import net.ontopia.topicmaps.query.core.QueryResultIF;
import net.ontopia.topicmaps.query.impl.basic.QueryMatches;
import net.ontopia.topicmaps.query.impl.utils.MultiCrossProduct;
import net.ontopia.topicmaps.query.impl.utils.QueryAnalyzer;
import net.ontopia.topicmaps.query.parser.AbstractClause;
import net.ontopia.topicmaps.query.parser.NotClause;
import net.ontopia.topicmaps.query.parser.OrClause;
import net.ontopia.topicmaps.query.parser.Pair;
import net.ontopia.topicmaps.query.parser.PredicateClause;
import net.ontopia.topicmaps.query.parser.TologQuery;
import net.ontopia.topicmaps.query.parser.Variable;
import net.ontopia.utils.OntopiaRuntimeException;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * INTERNAL: Class used to represent parsed queries. The class wraps a query
 * executer and a tolog query intance (as generated by the parser). The actual
 * query execution is delegated to the query executer.
 */
public final class ParsedQuery implements ParsedQueryIF {

  // Define a logging category.
  private static final Logger log = LoggerFactory.getLogger(ParsedQuery.class.getName());

  protected TologQuery query;

  protected net.ontopia.topicmaps.query.impl.rdbms.QueryProcessor rprocessor;

  protected net.ontopia.topicmaps.query.impl.basic.QueryProcessor bprocessor;

  protected QueryComponentIF[] components;

  protected QueryIF sqlquery;

  protected int qresult;

  protected boolean has_bclauses = false;

  protected SQLGeneratorIF sqlgen;

  public ParsedQuery(QueryProcessor rprocessor,
      net.ontopia.topicmaps.query.impl.basic.QueryProcessor bprocessor,
      TologQuery query) throws InvalidQueryException {

    // The rdbms query processor
    this.rprocessor = rprocessor;
    // The basic query processor
    this.bprocessor = bprocessor;
    // The tolog query
    this.query = query;

    // RULES:
    //
    // - If query only contains a NOT clause the result is always
    // empty.
    //
    // - Execute JDO predicates before BASIC predicates
    //

    // Create cross products of argument types
    MultiCrossProduct cp = new MultiCrossProduct(new Map[] {
        query.getVariableTypes(), query.getParameterTypes() });
    log.debug("Argument type cross product size: " + cp.getSize());
    
    if (cp.getSize() > 1) {
      // Pass query on to the basic processor
      this.has_bclauses = true;
      this.components = new QueryComponentIF[] { new BasicQueryComponent(query,
          query.getClauses(), this.bprocessor) };
      log
          .debug("Passing on query to basic.QueryProcessor because argument type cross product is "
              + cp.getSize() + " (>1).");

    } else if (cp.getSize() == 1) {
      // Cross product size is 1, so we'll give it a try
      while (cp.nextTuple()) {
        QueryBuilder builder = new QueryBuilder(query, rprocessor);
        builder.setVariables(cp.getMap(0));
        builder.setParameters(cp.getMap(1));
        // ! System.out.println("Variables: " + builder.getVariables());
        // ! System.out.println("Parameters: " + builder.getParameters());
        // Compile query
        compileQuery(builder, query);
        break;
      }
    } else {
      // No valid type combinations, so result must be false.
      this.qresult = -1;
    }
  }

  protected void compileQuery(QueryBuilder builder, TologQuery query)
      throws InvalidQueryException {
    // ! System.out.println("TOLOG: " + query);

    if (log.isDebugEnabled())
      log.debug("TOLOG: " + query);

    // TODO:
    //
    // 1. analyze query clauses and figure out possible variable types
    //
    // 2. weed out predicates and clauses that are have predictable results
    //
    // 3. make cross product for all variable type combinations
    //
    // 4. produce query filter for clause for each variable type binding
    //
    // 5. if there are multiple possible combinations put them in a
    // union-all expression, alternatively execute them serially.
    //
    // 6. bind the topicmap property to all the variables

    List qcomponents = new ArrayList();
    JDOQuery jdoquery = null;
    int bccount = 0;

    // Prescan query
    prescan(builder, query.getClauses());
      
    // Compile clauses
    QueryContext qcontext = compile(builder, query.getClauses());

    // Pass any non-mappable clauses to the basic processor
    if (qcontext.hasClauses()) {
      qcomponents.add(new BasicQueryComponent(query, qcontext.clauses,
          this.bprocessor));
      bccount++;
      this.has_bclauses = true;
      if (log.isDebugEnabled())
        log.debug("BASIC clauses: " + qcontext.clauses);
    }

    // Add JDO components when there are JDO expressions
    if (qcontext.hasExpressions()) {

      // not doing any aggregates or ordering if basic clauses exist
      boolean aggfunc = false;
      boolean orderby = false;

      if (!this.has_bclauses) {

        // TODO: weed out predicates that refer to variables without
        // 1. order by only not ordered by TopicIF name
        orderby = !query.getOrderBy().isEmpty() && isOrderableTypes(query);

        // 2. aggregate functions only when no basic clauses
        aggfunc = !query.getCountedVariables().isEmpty();

        // ! // add basic reduce component
        // ! qcomponents.add(new BasicReduceComponent(query, this.bprocessor));
        // // NEEDED?

        // ! // FIXME: Cannot order topic variables correctly according to
        // ! // current tolog rules. This particularly applies to TopicIFs,
        // ! // since it is expected that they are ordered by name. So if
        // ! // ordered variable is not counted (ie. a number) we leave the
        // ! // ordering to the basic sort component.
        // ! for (int ix = 0; ix < oblist.size(); ix++) {
        // ! Variable var = (Variable)oblist.get(ix);
        // !
        // ! // TODO: only disable orderby if all variables are TopicIFs
        // ! if (!counted.contains(var)) {
        // ! // JDO query is not orderable
        // ! orderby = false;
        // ! break;
        // ! }
        // ! }

        // ! // add basic count component
        // ! if (!aggfunc && !counted.isEmpty()) {
        // ! qcomponents.add(new BasicCountComponent(query, this.bprocessor));
        // ! bccount++;
        // ! }

        // add basic sort component
        if (!orderby && !query.getOrderBy().isEmpty()) {
          qcomponents.add(new BasicSortComponent(query, this.bprocessor));
          bccount++;
        }
      }

      // Compile JDO query
      jdoquery = makeJDOQuery(builder, qcontext, aggfunc, orderby);
      if (log.isDebugEnabled())
        log.debug("JDO: " + jdoquery + " [vars: " + jdoquery.getVariableCount()
            + "]");

      // Evaluate JDO query and reduce query filter
      this.qresult = JDOEvaluator.evaluateExpression(jdoquery.getFilter(),
          this.rprocessor.getMapping(), true);
      if (log.isDebugEnabled())
        log.debug("JDO: evaluation result is " + this.qresult);
    }

    // RDBMS tolog only
    if (bccount == 0 && this.qresult == 0 && jdoquery != null) {
      // FIXME: May not want to wrap in matrix query, but read directly
      // into QueryMatches object.

      RDBMSStorage storage = (RDBMSStorage) this.rprocessor.getTransaction()
          .getStorageAccess().getStorage();
      this.sqlgen = storage.getSQLGenerator();

      // Set limit and offset
      if (this.sqlgen.supportsLimitOffset()) {
        jdoquery.setLimit(query.getLimit());
        jdoquery.setOffset(query.getOffset());
      }

      // Query is non-evaluatable and thus SQL executable
      this.sqlquery = this.rprocessor.getTransaction().createQuery(jdoquery,
          true);
    }

    // BASIC and RDBMS tolog
    else {
      if (jdoquery != null && this.qresult == 0) {
        // Make sure JDO component is evaluated first
        QueryIF matrix = this.rprocessor.getTransaction().createQuery(jdoquery, true);

        String[] colnames = jdoquery.getSelectedColumnNames();
        qcomponents.add(0, new JDOQueryComponent(matrix, colnames));
      }
      this.components = new QueryComponentIF[qcomponents.size()];
      qcomponents.toArray(this.components);
    }
  }

  class JDONamedAggregator implements JDOVisitorIF {

    protected Set varnames = new HashSet(5);
    protected Set parnames = new HashSet(5);

    @Override
    public void visitable(JDOExpressionIF expr) {
      expr.visit(this);
    }

    @Override
    public void visitable(JDOExpressionIF[] exprs) {
      for (int i = 0; i < exprs.length; i++) {
        exprs[i].visit(this);
      }
    }

    @Override
    public void visitable(JDOValueIF value) {
      if (value.getType() == JDOValueIF.VARIABLE) {
        varnames.add(((JDOVariable) value).getName());
      } else if (value.getType() == JDOValueIF.PARAMETER) {
        parnames.add(((JDOParameter) value).getName());
      }
      value.visit(this);
    }

    @Override
    public void visitable(JDOValueIF[] values) {
      for (int i = 0; i < values.length; i++) {
        visitable(values[i]);
      }
    }
  }

  protected JDOQuery makeJDOQuery(QueryBuilder builder, QueryContext qcontext,
      boolean aggfunc, boolean orderby) throws InvalidQueryException {
    // Do we have any top level expressions?
    if (qcontext.hasExpressions()) {

      // TODO: Do we really have to add topic map filtering expression
      // here? Make sure query is bound to the current topic map.
      //
      // Would it be better to register this stuff with QueryBuilder
      // instead?

      JDONamedAggregator visitor = new JDONamedAggregator();
      Iterator iter1 = qcontext.expressions.iterator();
      while (iter1.hasNext()) {
        ((JDOExpressionIF) iter1.next()).visit(visitor);
        ;
      }

      // Create query for AND'ed expressions
      JDOQuery jdoquery = new JDOQuery();

      // Register variables
      Iterator iter2 = visitor.varnames.iterator();
      while (iter2.hasNext()) {
        String varname = (String) iter2.next();
        Class vartype = builder.getVariableType(varname);
        if (vartype == null)
          throw new InvalidQueryException(
              "Not able to figure out type of variable: $" + varname);
        jdoquery.addVariable(varname, vartype);
      }

      // Register parameters
      Iterator iter3 = visitor.parnames.iterator();
      while (iter3.hasNext()) {
        String parname = (String) iter3.next();
        Class partype = builder.getParameterType(parname);
        if (partype == null)
          throw new InvalidQueryException(
              "Not able to figure out type of parameter: %" + parname + '%');
        jdoquery.addParameter(parname, partype);
      }

      // ! builder.registerJDOVariables(jdoquery);
      // ! builder.registerJDOParameters(jdoquery);

      // ! System.out.println("VARS: " + jdoquery.getVariableNames());
      // ! System.out.println("PARAMS: " + jdoquery.getParameterNames());

      // TODO: Add support for the DISTINCT keyword
      jdoquery.setDistinct(true);

      // ISSUE: no need to do distinct if we have basic clauses
      // ! jdoquery.setDistinct(!this.has_bclauses || !orderby);
      // ! System.out.println("X: " + jdoquery.getDistinct() + " " +
      // this.has_bclauses + " " + orderby);

      // Register select
      if (qcontext.hasClauses())
        builder.registerJDOSelectDependent(jdoquery, visitor.varnames);
      else
        builder.registerJDOSelect(jdoquery, visitor.varnames, aggfunc);

      // Register order by
      if (orderby)
        builder.registerJDOOrderBy(jdoquery, aggfunc);

      jdoquery.setFilter(new JDOAnd(qcontext.expressions));
      return jdoquery;
    } else
      return null;
  }

  public List getClauses() {
    return query.getClauses();
  }

  // / ParsedQueryIF implementation [the class does not implement the interface]

  @Override
  public List getSelectedVariables() {
    return getVariables(query.getSelectedVariables());
  }

  @Override
  public Collection getAllVariables() {
    return getVariables(query.getAllVariables());
  }

  @Override
  public Collection getCountedVariables() {
    return getVariables(query.getCountedVariables());
  }

  @Override
  public List getOrderBy() {
    return getVariables(query.getOrderBy());
  }

  @Override
  public boolean isOrderedAscending(String name) {
    return query.isOrderedAscending(name);
  }

  protected List getVariables(Collection varnames) {
    List results = new ArrayList(varnames.size());
    Iterator iter = varnames.iterator();
    while (iter.hasNext()) {
      results.add(((Variable) iter.next()).getName());
    }
    return results;
  }

  @Override
  public QueryResultIF execute() throws InvalidQueryException {
    return execute(null);
  }

  @Override
  public QueryResultIF execute(Map arguments) throws InvalidQueryException {
    // sanity-check arguments
    QueryAnalyzer.verifyParameters(query, arguments);

    // flush transaction, need to make sure that all dirty data is stored
    rprocessor.getTransaction().flush();

    if (this.sqlquery != null) {
      try {

        // since all clauses mapped to native query equivalents we can
        // execute the entire query in one go and return the result,
        // with having to go via a QueryMatches instance.
        net.ontopia.persistence.proxy.QueryResultIF qresult =
        // execute with named arguments
        (net.ontopia.persistence.proxy.QueryResultIF) (arguments == null ? this.sqlquery
            .executeQuery()
            : this.sqlquery.executeQuery(arguments));

        // figure out if OFFSET/LIMIT is to be done in resultset or natively in
        // database
        if (this.sqlgen != null && !this.sqlgen.supportsLimitOffset()) {
          return new QueryResult(qresult, query.getSelectedVariableNames(),
              query.getLimit(), query.getOffset());
        } else {
          return new QueryResult(qresult, query.getSelectedVariableNames());
        }
      } catch (Exception e) {
        throw new OntopiaRuntimeException(e);
      }
    } else {
      // if query succeed its pre-evaluation return a boolean result.
      if (this.qresult == 1)
        return new BooleanQueryResult(query.getSelectedVariableNames(), true);
      else if (this.qresult == -1)
        return new BooleanQueryResult(query.getSelectedVariableNames(), false);

      // ISSUE: Do we need to do anything with the arguments here?
      // TODO: This is _not_ thread safe

      // set query arguments
      if (arguments != null)
        query.setArguments(arguments);

      // prepare query matches instance
      QueryMatches matches = prepareQueryMatches(arguments);

      if (log.isDebugEnabled())
        log.debug("Components: " + Arrays.asList(components));

      // loop over query components and let them process the query matches.
      if (this.components != null) {
        for (int i = 0; i < this.components.length; i++) {
          matches = this.components[i].satisfy(matches, arguments);
        }
      }

      // clear query arguments
      if (arguments != null)
        query.setArguments(null);

      // wrap QueryMatches in a QueryResultIF.
      return new net.ontopia.topicmaps.query.impl.basic.QueryResult(matches,
          query.getLimit(), query.getOffset());
    }
  }

  protected QueryMatches prepareQueryMatches(Map arguments) {
    if (this.has_bclauses)
      return bprocessor.createInitialMatches(query, arguments);
    else
      return bprocessor.createInitialMatches(query, query
          .getSelectedVariables(), arguments);
  }

  class QueryContext {

    protected List clauses = new ArrayList();

    protected List expressions = new ArrayList();

    public boolean hasClauses() {
      return !clauses.isEmpty();
    }

    public boolean hasExpressions() {
      return !expressions.isEmpty();
    }
  }

  protected void prescan(QueryBuilder builder, List clauses) {

    // detect variables that map to multiple columns    
    for (int ix = 0; ix < clauses.size(); ix++) {
      AbstractClause theClause = (AbstractClause) clauses.get(ix);

      if (theClause instanceof PredicateClause) {
        PredicateClause pclause = (PredicateClause) theClause;
        if (pclause.getPredicate() instanceof JDOPredicateIF) {
          JDOPredicateIF pred = (JDOPredicateIF) pclause.getPredicate();
          pred.prescan(builder, pclause.getArguments());
        }

      } else if (theClause instanceof OrClause) {
        OrClause clause = (OrClause) theClause;
        List alternatives = clause.getAlternatives();
        int len = alternatives.size();
        for (int i = 0; i < len; i++) {
          prescan(builder, (List) alternatives.get(i));
        }

      } else if (theClause instanceof NotClause) {
        NotClause clause = (NotClause) theClause;
        prescan(builder, clause.getClauses());

      } else
        throw new OntopiaRuntimeException("Unknown clause type:" + theClause.getClass());
    }
  }

  protected boolean isSupportedArguments(QueryBuilder builder, List arguments) {
    int len = arguments.size();
    for (int i=0; i < len; i++) {
      Object arg = arguments.get(i);
      if (arg instanceof Variable
          && !builder.isSupportedVariable((Variable)arg))
        return false;
      else if (arg instanceof Pair) {
        Pair pair = (Pair)arg;
        if (pair.getFirst() instanceof Variable
            && !builder.isSupportedVariable((Variable)pair.getFirst()))
          return false;
        if (pair.getSecond() instanceof Variable
            && !builder.isSupportedVariable((Variable)pair.getSecond()))
          return false;
      }
    }
    return true;
  }
  
  protected QueryContext compile(QueryBuilder builder, List clauses)
      throws InvalidQueryException {
    QueryContext qcontext = new QueryContext();

    for (int ix = 0; ix < clauses.size(); ix++) {
      AbstractClause theClause = (AbstractClause) clauses.get(ix);

      if (theClause instanceof PredicateClause) {
        PredicateClause pclause = (PredicateClause) theClause;
        // FIXME: If predicate is recursive we need to break up the
        // query at this point.
        if (pclause.getPredicate() instanceof JDOPredicateIF) {
          JDOPredicateIF pred = (JDOPredicateIF) pclause.getPredicate();

          // Either create JDOExpressionIFs or embed in basic executer
          List args = pclause.getArguments();
          boolean hadexpr = (isSupportedArguments(builder, args) &&
                             pred.buildQuery(builder, qcontext.expressions, args));
          if (!hadexpr) 
            qcontext.clauses.add(pclause);
        } else {
          // Predicate not an rdbms-tolog predicate, so we'll evaluate it with
          // basic-tolog instead.
          qcontext.clauses.add(pclause);
        }

      } else if (theClause instanceof OrClause) {
        OrClause clause = (OrClause) theClause;

        boolean broken = false;
        List exprs = new ArrayList();
        List alternatives = clause.getAlternatives();
        int len = alternatives.size();
        if (len == 1 || clause.getShortCircuit())
          // optional clause
          broken = true;
        else {
          // ordinary OR
          for (int i = 0; !broken && i < len; i++) {
            QueryContext _qcontext = compile(builder, (List) alternatives
                .get(i));

            if (_qcontext.hasClauses())
              broken = true;
            else
              exprs.add(new JDOAnd(_qcontext.expressions));
          }
        }

        // Map expression
        if (broken)
          qcontext.clauses.add(clause);
        else
          qcontext.expressions.add(new JDOOr(exprs));

      } else if (theClause instanceof NotClause) {
        NotClause clause = (NotClause) theClause;

        // TODO: Variables should be local to the JDONot in this
        // case. Unfortunately they get lost at this time, because the
        // query builder used is discarded.
        //
        // Variables have to be put onto a stack, so that one can
        // restrict their "validity".

        // Compile NOT clause
        QueryContext _qcontext = compile(builder, clause.getClauses());

        // Map expression
        if (!_qcontext.hasClauses() && _qcontext.hasExpressions())
          qcontext.expressions
              .add(new JDONot(new JDOAnd(_qcontext.expressions)));
        else
          qcontext.clauses.add(clause);

      } else
        throw new OntopiaRuntimeException("Unknown clause type:"
            + theClause.getClass());

    }

    // If qcontext.expressions contains only non-independent clauses
    // (e.g. NOTs) then let basic processor process them.
    int size = qcontext.expressions.size();
    if (size > 0) {
      boolean indeps = false;
      for (int i = 0; i < size; i++) {
        JDOExpressionIF expr = (JDOExpressionIF) qcontext.expressions.get(i);
        if (isIndependent(expr)) {
          indeps = true;
          break;
        }
      }
      if (!indeps) {
        qcontext.clauses = clauses;
        qcontext.expressions.clear();
      }
    }

    return qcontext;
  }

  protected boolean isIndependent(JDOExpressionIF expr) {
    // Note: This is a crude way of avoiding queries with too few
    // filtering expressions. It will also avoid the case when the query
    // only contains JDONots.

    switch (expr.getType()) {
    case JDOExpressionIF.AND: {
      JDOExpressionIF[] exprs = ((JDOAnd) expr).getExpressions();
      for (int i = 0; i < exprs.length; i++) {
        if (isIndependent(exprs[i]))
          return true;
        ;
      }
      return false;
    }
    case JDOExpressionIF.OR: {
      JDOExpressionIF[] exprs = ((JDOOr) expr).getExpressions();
      for (int i = 0; i < exprs.length; i++) {
        if (isIndependent(exprs[i]))
          return true;
        ;
      }
    }
    // case JDOExpressionIF.NOT_EQUAL:
    case JDOExpressionIF.NOT: {
      return false;
      // ! return isIndependent(((JDONot)expr).getExpression());
    }
    }
    // All other expressions are independent.
    return true;
  }

  protected boolean isOrderableTypes(TologQuery query) {
    List oblist = query.getOrderBy();
    int size = oblist.size();
    for (int i = 0; i < size; i++) {
      Variable var = (Variable) oblist.get(i);
      String varname = var.getName();
      Object[] vartypes = (Object[]) query.getVariableTypes().get(varname);
      for (int x = 0; x < vartypes.length; x++) {
        if (net.ontopia.topicmaps.core.TopicIF.class.equals(vartypes[x])
            && !query.getCountedVariables().contains(var))
          return false;
      }
    }
    return true;
  }

  // / java.lang.Object implementation

  @Override
  public String toString() {
    return query.toString();
  }

}
